{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "solving pendulum using actor-critic model\n",
    "\"\"\"\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.layers import Dense, Dropout, Input, GRU, Masking\n",
    "from tensorflow.keras.layers import Add, Concatenate\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import random\n",
    "from collections import deque\n",
    "\n",
    "def stack_samples(samples):\n",
    "\tarray = np.array(samples)\n",
    "\t\n",
    "\tcurrent_states = np.stack(array[:,0]).reshape((array.shape[0],-1,array[0,0].shape[2]))\n",
    "\tcurrent_act_hists = np.stack(array[:,1]).reshape((array.shape[0],-1,array[0,1].shape[2]))\n",
    "\tactions = np.stack(array[:,2]).reshape((array.shape[0],-1))\n",
    "\trewards = np.stack(array[:,3]).reshape((array.shape[0],-1))\n",
    "\tnew_states = np.stack(array[:,4]).reshape((array.shape[0],-1,array[0,4].shape[2]))\n",
    "\tnew_act_hists = np.stack(array[:,5]).reshape((array.shape[0],-1,array[0,5].shape[2]))\n",
    "\tdones = np.stack(array[:,6]).reshape((array.shape[0],1))\n",
    "\t\n",
    "\treturn current_states, current_act_hists, actions, rewards, new_states, new_act_hists, dones\n",
    "\n",
    "# determines how to assign values to each state, i.e. takes the state\n",
    "# and action (two-input model) and determines the corresponding value\n",
    "class ActorCritic:\n",
    "\tdef __init__(self, env, sess):\n",
    "\t\tself.env  = env\n",
    "\t\tself.sess = sess\n",
    "\n",
    "\t\tself.learning_rate = 0.0001\n",
    "\t\tself.epsilon = .9\n",
    "\t\tself.epsilon_decay = .99995\n",
    "\t\tself.gamma = .90\n",
    "\t\tself.tau   = .01\n",
    "\n",
    "\t\t# ===================================================================== #\n",
    "\t\t#                               Actor Model                             #\n",
    "\t\t# Chain rule: find the gradient of chaging the actor network params in  #\n",
    "\t\t# getting closest to the final value network predictions, i.e. de/dA    #\n",
    "\t\t# Calculate de/dA as = de/dC * dC/dA, where e is error, C critic, A act #\n",
    "\t\t# ===================================================================== #\n",
    "\t\tself.ema = tf.train.ExponentialMovingAverage(decay=1-self.tau)\n",
    "\t\tself.memory = deque(maxlen=4000)\n",
    "\t\tself.actor_state_input, self.actor_act_hist_input, self.actor_model = self.create_actor_model()\n",
    "\t\t_,_, self.target_actor_model = self.create_actor_model()\n",
    "\n",
    "\t\tself.actor_critic_grad = tf.placeholder(tf.float32,\n",
    "\t\t\t[None, self.env.action_space.shape[0]]) # where we will feed de/dC (from critic)\n",
    "\n",
    "\t\tactor_model_weights = self.actor_model.trainable_weights\n",
    "\t\tself.actor_grads = tf.gradients(self.actor_model.output,\n",
    "\t\t\tactor_model_weights, -self.actor_critic_grad) # dC/dA (from actor)\n",
    "\t\tgrads = zip(self.actor_grads, actor_model_weights)\n",
    "\t\tself.optimize = tf.train.AdamOptimizer(self.learning_rate).apply_gradients(grads)\n",
    "\n",
    "\t\t# ===================================================================== #\n",
    "\t\t#                              Critic Model                             #\n",
    "\t\t# ===================================================================== #\n",
    "\n",
    "\t\tself.critic_state_input, self.critic_act_hist_input, self.critic_action_input, \\\n",
    "\t\t\tself.critic_model = self.create_critic_model()\n",
    "\t\t_, _, _,self.target_critic_model = self.create_critic_model()\n",
    "\n",
    "\t\tself.critic_grads = tf.gradients(self.critic_model.output,\n",
    "\t\t\tself.critic_action_input) # where we calcaulte de/dC for feeding above\n",
    "\n",
    "\t\t# Initialize for later gradient calculations\n",
    "\t\tself.sess.run(tf.initialize_all_variables())\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                              Model Definitions                            #\n",
    "\n",
    "\tdef create_actor_model(self):\n",
    "\t# ========================================================================= #\n",
    "\t\tstate_input = Input(shape=(None,self.env.observation_space.shape[0]))\n",
    "\t\tact_hist_input = Input(shape=(None,self.env.action_space.shape[0]))\n",
    "\t\tmask_state_input = Masking(mask_value=0.)(state_input)\n",
    "\t\tmask_action_input = Masking(mask_value=0.)(act_hist_input)\n",
    "\t\th1 = Dense(500, activation='relu')(Concatenate()([mask_state_input,mask_action_input]))\n",
    "\t\tactor_rnn,state_h = GRU(256, return_state=True)(h1)\n",
    "\t\th2 = Dense(500, activation='relu')(state_h)\n",
    "\t\toutput = Dense(self.env.action_space.shape[0], activation='tanh')(h2)\n",
    "\n",
    "\t\tmodel = Model([state_input,act_hist_input], output)\n",
    "\t\tadam  = Adam(lr=0.0001)\n",
    "\t\tmodel.compile(loss=\"mse\", optimizer=adam)\n",
    "\t\treturn state_input, act_hist_input, model\n",
    "\n",
    "\tdef create_critic_model(self):\n",
    "\t\tstate_input = Input(shape=(None,self.env.observation_space.shape[0]))\n",
    "\t\tact_hist_input = Input(shape=(None,self.env.action_space.shape[0]))\n",
    "\t\tmask_state_input = Masking(mask_value=0.)(state_input)\n",
    "\t\tmask_action_input = Masking(mask_value=0.)(act_hist_input)\n",
    "\t\tstate_h1 = Dense(500, activation='relu')(Concatenate()([mask_state_input,mask_action_input]))\n",
    "\t\tcritic_rnn,state_h2 = GRU(256, return_state=True)(state_h1)\n",
    "\n",
    "\t\taction_input = Input(shape=self.env.action_space.shape)\n",
    "\t\taction_h1    = Dense(500)(action_input)\n",
    "\n",
    "\t\tmerged    = Concatenate()([state_h2, action_h1])\n",
    "\t\tmerged_h1 = Dense(500, activation='relu')(merged)\n",
    "\t\toutput = Dense(1, activation='linear')(merged_h1)\n",
    "\t\tmodel  = Model([state_input,act_hist_input,action_input],output)\n",
    "\n",
    "\t\tadam  = Adam(lr=0.0001)\n",
    "\t\tmodel.compile(loss=\"mse\", optimizer=adam)\n",
    "\t\treturn state_input, act_hist_input, action_input, model\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                               Model Training                              #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef remember(self, cur_state, cur_act_hist, action, reward, new_state, new_act_hist, done):\n",
    "\t\tself.memory.append([cur_state, cur_act_hist, action, reward, new_state, new_act_hist, done])\n",
    "\n",
    "\tdef _train_actor(self, samples):\n",
    "\t\t\n",
    "\t\t\tcur_states, cur_act_hists, actions, rewards, new_states,new_act_hists, _ =  stack_samples(samples)\n",
    "\t\t\tpredicted_actions = self.actor_model.predict([cur_states,cur_act_hists])\n",
    "\t\t\tgrads = self.sess.run(self.critic_grads, feed_dict={\n",
    "\t\t\t\tself.critic_state_input:  cur_states,\n",
    "                self.critic_act_hist_input: cur_act_hists,\n",
    "\t\t\t\tself.critic_action_input: predicted_actions\n",
    "\t\t\t})[0]\n",
    "\n",
    "\t\t\tself.sess.run(self.optimize, feed_dict={\n",
    "\t\t\t\tself.actor_state_input: cur_states,\n",
    "                self.actor_act_hist_input: cur_act_hists,\n",
    "\t\t\t\tself.actor_critic_grad: grads\n",
    "\t\t\t})\n",
    "\n",
    "\tdef _train_critic(self, samples):\n",
    "   \n",
    "\n",
    "\t\tcur_states, cur_act_hists, actions, rewards, new_states,new_act_hists, dones = stack_samples(samples)\n",
    "\t\ttarget_actions = self.target_actor_model.predict([new_states,new_act_hists])\n",
    "\t\tfuture_rewards = self.target_critic_model.predict([new_states, new_act_hists,target_actions])\n",
    "\t\tdones = dones.reshape(rewards.shape)\n",
    "\t\tfuture_rewards = future_rewards.reshape(rewards.shape)\n",
    "\t\trewards += self.gamma * future_rewards * (1 - dones)\n",
    "\t\t\n",
    "\t\tevaluation = self.critic_model.fit([cur_states, cur_act_hists, actions], rewards, verbose=0)\n",
    "\t\t#print(evaluation.history)\n",
    "        \n",
    "\tdef train(self):\n",
    "\t\tbatch_size = 256\n",
    "\t\tif len(self.memory) < batch_size:\n",
    "\t\t\treturn\n",
    "\n",
    "\t\trewards = []\n",
    "\t\tsamples = random.sample(self.memory, batch_size)\n",
    "\t\tself.samples = samples\n",
    "\t\tself._train_critic(samples)\n",
    "\t\tself._train_actor(samples)\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                         Target Model Updating                             #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef _update_actor_target(self):\n",
    "\t\tactor_model_weights  = self.actor_model.get_weights()\n",
    "\t\tactor_target_weights = self.target_actor_model.get_weights()\n",
    "\t\t\n",
    "\t\tfor i in range(len(actor_target_weights)):\n",
    "\t\t\tactor_target_weights[i] = actor_model_weights[i]*self.tau + actor_target_weights[i]*(1-self.tau)\n",
    "\t\tself.target_actor_model.set_weights(actor_target_weights)\n",
    "\n",
    "\tdef _update_critic_target(self):\n",
    "\t\tcritic_model_weights  = self.critic_model.get_weights()\n",
    "\t\tcritic_target_weights = self.target_critic_model.get_weights()\n",
    "\t\t\n",
    "\t\tfor i in range(len(critic_target_weights)):\n",
    "\t\t\tcritic_target_weights[i] = critic_model_weights[i]*self.tau + critic_target_weights[i]*(1-self.tau)\n",
    "\t\tself.target_critic_model.set_weights(critic_target_weights)\n",
    "\n",
    "\tdef update_target(self):\n",
    "\t\tself._update_actor_target()\n",
    "\t\tself._update_critic_target()\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                              Model Predictions                            #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef act(self, cur_state, cur_act_hist):\n",
    "\t\tself.epsilon *= self.epsilon_decay\n",
    "\t\tif np.random.random() < self.epsilon:\n",
    "\t\t\treturn self.actor_model.predict([cur_state,cur_act_hist])*2 + np.random.normal()\n",
    "\t\treturn self.actor_model.predict([cur_state,cur_act_hist])*2\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py:3794: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/tf_should_use.py:193: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "trial:0\n",
      "[-13.053879]\n",
      "trial:1\n",
      "[-18.335472]\n",
      "trial:2\n",
      "[-10.345471]\n",
      "trial:3\n",
      "[-6.6736894]\n",
      "trial:4\n",
      "[-12.528373]\n",
      "trial:5\n",
      "[-13.061871]\n",
      "trial:6\n",
      "[-17.911516]\n",
      "trial:7\n",
      "[-4.9956036]\n",
      "trial:8\n",
      "[-11.904888]\n",
      "trial:9\n",
      "[-0.8698223]\n",
      "trial:10\n",
      "[-18.462803]\n",
      "trial:11\n",
      "[-13.893444]\n",
      "trial:12\n",
      "[-13.916513]\n",
      "trial:13\n",
      "[-16.861692]\n",
      "trial:14\n",
      "[-16.493332]\n",
      "trial:15\n",
      "[-19.114054]\n",
      "trial:16\n",
      "[-12.455459]\n",
      "trial:17\n",
      "[-14.736842]\n",
      "trial:18\n",
      "[-16.045942]\n",
      "trial:19\n",
      "[-16.163212]\n",
      "trial:20\n",
      "[-14.488147]\n",
      "trial:21\n",
      "[-14.85277]\n",
      "trial:22\n",
      "[-11.448878]\n",
      "trial:23\n",
      "[-13.977213]\n",
      "trial:24\n",
      "[-15.237435]\n",
      "trial:25\n",
      "[-10.873445]\n",
      "trial:26\n",
      "[-13.530369]\n",
      "trial:27\n",
      "[-14.158274]\n",
      "trial:28\n",
      "[-1.3639754]\n",
      "trial:29\n",
      "[-4.245361]\n",
      "trial:30\n",
      "[-9.153479]\n",
      "trial:31\n",
      "[-9.404124]\n",
      "trial:32\n",
      "[-6.9807515]\n",
      "trial:33\n",
      "[-10.097956]\n",
      "trial:34\n",
      "[-1.1311238]\n",
      "trial:35\n",
      "[-23.647028]\n",
      "trial:36\n",
      "[-0.3736391]\n",
      "trial:37\n",
      "[-0.05426612]\n",
      "trial:38\n",
      "[-22.184923]\n",
      "trial:39\n",
      "[-14.257694]\n",
      "trial:40\n",
      "[-11.914403]\n",
      "trial:41\n",
      "[-20.071478]\n",
      "trial:42\n",
      "[-0.1076529]\n",
      "trial:43\n",
      "[-0.00262552]\n",
      "trial:44\n",
      "[-0.15596034]\n",
      "trial:45\n",
      "[-0.00141638]\n",
      "trial:46\n",
      "[-0.01549252]\n",
      "trial:47\n",
      "[-0.00729494]\n",
      "trial:48\n",
      "[-0.00299818]\n",
      "trial:49\n",
      "[-0.00556854]\n",
      "trial:50\n",
      "[-0.00908865]\n",
      "trial:51\n",
      "[-0.10848352]\n",
      "trial:52\n",
      "[-0.02825909]\n",
      "trial:53\n",
      "[-0.00878663]\n",
      "trial:54\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-c13c789c4a8d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"__main__\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m         \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-2-c13c789c4a8d>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mj\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m5\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m                                 \u001b[0mactor_critic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m                                 \u001b[0mactor_critic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-1-fdc072442d97>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    159\u001b[0m                 \u001b[0msamples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    160\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msamples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msamples\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 161\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_critic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    162\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_actor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-1-fdc072442d97>\u001b[0m in \u001b[0;36m_train_critic\u001b[0;34m(self, samples)\u001b[0m\n\u001b[1;32m    148\u001b[0m                 \u001b[0mrewards\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgamma\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mfuture_rewards\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mdones\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m                 \u001b[0mevaluation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcur_states\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcur_act_hists\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactions\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrewards\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m                 \u001b[0;31m#print(evaluation.history)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    778\u001b[0m           \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    779\u001b[0m           \u001b[0mvalidation_freq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_freq\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 780\u001b[0;31m           steps_name='steps_per_epoch')\n\u001b[0m\u001b[1;32m    781\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    782\u001b[0m   def evaluate(self,\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq, mode, validation_in_fit, prepared_feed_values_from_dataset, steps_name, **kwargs)\u001b[0m\n\u001b[1;32m    361\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    362\u001b[0m         \u001b[0;31m# Get outputs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 363\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    364\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    365\u001b[0m           \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3290\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3291\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[0;32m-> 3292\u001b[0;31m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[1;32m   3293\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3294\u001b[0m     output_structure = nest.pack_sequence_as(\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1456\u001b[0m         ret = tf_session.TF_SessionRunCallable(self._session._session,\n\u001b[1;32m   1457\u001b[0m                                                \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1458\u001b[0;31m                                                run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1459\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1460\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def main():\n",
    "\tsess = tf.Session()\n",
    "\tK.set_session(sess)\n",
    "\tenv = gym.make(\"Pendulum-v0\")\n",
    "\tactor_critic = ActorCritic(env, sess)\n",
    "\n",
    "\tnum_trials = 10000\n",
    "\ttrial_len  = 200\n",
    "\n",
    "\tfor i in range(num_trials):\n",
    "\t\tprint(\"trial:\" + str(i))\n",
    "\t\tcur_state = env.reset()\n",
    "\t\taction = env.action_space.sample()\n",
    "\t\treward_sum = 0\n",
    "\t\tobs_list = []      \n",
    "\t\tact_list = []\n",
    "\t\tobs_list.append(cur_state)\n",
    "\t\tact_list.append(np.zeros((1,actor_critic.env.action_space.shape[0])))\n",
    "\t\tfor j in range(trial_len):\n",
    "\t\t\t#env.render()        \n",
    "\t\t\tobs_seq = np.asarray(obs_list)\n",
    "\t\t\tact_seq = np.asarray(act_list)\n",
    "\t\t\tobs_seq = obs_seq.reshape((1, -1, env.observation_space.shape[0]))\n",
    "\t\t\tact_seq = act_seq.reshape((1, -1, env.action_space.shape[0]))\n",
    "\t\t\taction = actor_critic.act(obs_seq,act_seq)\n",
    "\t\t\taction = action.reshape((1, env.action_space.shape[0]))\n",
    "\n",
    "\t\t\tnew_state, reward, done, _ = env.step(action)\n",
    "\t\t\treward_sum += reward\n",
    "            reward+=reward\n",
    "\t\t\tif j == (trial_len - 1):\n",
    "\t\t\t\tdone = True\n",
    "\t\t\t\tprint(reward_sum)\n",
    "\n",
    "\t\t\tif (j % 5 == 0):\n",
    "\t\t\t\tactor_critic.train()\n",
    "\t\t\t\tactor_critic.update_target()   \n",
    "\t\t\t\n",
    "\t\t\tnew_state = new_state.reshape((env.observation_space.shape))\n",
    "\t\t\taction = action.reshape((env.action_space.shape))\n",
    "\n",
    "\t\t\tobs_list.append(new_state)\n",
    "\t\t\tact_list.append(action)\n",
    "\t\t\tnext_obs_seq = np.asarray(obs_list)\n",
    "\t\t\tnext_act_seq = np.asarray(act_list)\n",
    "\t\t\tnext_obs_seq = next_obs_seq.reshape((1, -1, env.observation_space.shape[0]))\n",
    "\t\t\tnext_act_seq = next_act_seq.reshape((1, -1, env.action_space.shape[0]))\n",
    "            \n",
    "\t\t\t#padding\n",
    "\t\t\tpad_width = trial_len-np.size(obs_seq,1)\n",
    "\t\t\tobs_seq = np.pad(obs_seq,((0,0),(pad_width,0),(0,0)))\n",
    "\t\t\tnext_obs_seq = np.pad(next_obs_seq,((0,0),(pad_width,0),(0,0)))\n",
    "\t\t\tact_seq = np.pad(act_seq,((0,0),(pad_width,0),(0,0)))\n",
    "\t\t\tnext_act_seq = np.pad(next_act_seq,((0,0),(pad_width,0),(0,0)))\n",
    "\t\t\t#print(obs_seq.shape)\n",
    "\t\t\t#print(next_obs_seq.shape)\n",
    "            \n",
    "\t\t\tactor_critic.remember(obs_seq, act_seq, action, reward, next_obs_seq, next_act_seq, done)\n",
    "\t\t\tcur_state = new_state\n",
    "\n",
    "\t\tif (i % 5 == 0) and i!=0:\n",
    "\t\t\tcur_state = env.reset()\n",
    "\t\t\tobs_list = []         \n",
    "\t\t\tact_list = []\n",
    "\t\t\tact_list.append(np.zeros((1,env.action_space.shape[0])))\n",
    "\t\t\tfor j in range(500):\n",
    "\t\t\t\tenv.render()\n",
    "\t\t\t\tcur_state = cur_state.reshape((1, env.observation_space.shape[0]))\n",
    "\t\t\t\tobs_list.append(cur_state)\n",
    "\t\t\t\tobs_seq = np.asarray(obs_list)\n",
    "\t\t\t\tact_seq = np.asarray(act_list)\n",
    "\t\t\t\tobs_seq = obs_seq.reshape((1, -1, env.observation_space.shape[0]))\n",
    "\t\t\t\tact_seq = act_seq.reshape((1, -1, env.action_space.shape[0]))\n",
    "\t\t\t\taction = actor_critic.act(obs_seq,act_seq)\n",
    "\t\t\t\taction = action.reshape((1, env.action_space.shape[0]))\n",
    "\n",
    "\t\t\t\tnew_state, reward, done, _ = env.step(action)\n",
    "\t\t\t\t#reward += reward\n",
    "\t\t\t\t#if j == (trial_len - 1):\n",
    "\t\t\t\t\t#done = True\n",
    "\t\t\t\t\t#print(reward)\n",
    "\n",
    "\t\t\t\t#if (j % 5 == 0):\n",
    "\t\t\t\t#    actor_critic.train()\n",
    "\t\t\t\t#    actor_critic.update_target()   \n",
    "\t\t\t\t\n",
    "\t\t\t\tnew_state = new_state.reshape((1, env.observation_space.shape[0]))\n",
    "\n",
    "\t\t\t\t#actor_critic.remember(cur_state, action, reward, new_state, done)\n",
    "\t\t\t\tcur_state = new_state\n",
    "\t\t\t\tact_list.append(action)\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\tmain()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'env' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-6d356fa9acfd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcur_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mobs_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mact_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mact_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m500\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'env' is not defined"
     ]
    }
   ],
   "source": [
    "cur_state = env.reset()\n",
    "obs_list = []         \n",
    "act_list = []\n",
    "act_list.append(np.zeros((1,env.action_space.shape[0])))\n",
    "for j in range(500):\n",
    "    env.render()\n",
    "    cur_state = cur_state.reshape((1, env.observation_space.shape[0]))\n",
    "    obs_list.append(cur_state)\n",
    "    obs_seq = np.asarray(obs_list)\n",
    "    act_seq = np.asarray(act_list)\n",
    "    obs_seq = obs_seq.reshape((1, -1, env.observation_space.shape[0]))\n",
    "    act_seq = act_seq.reshape((1, -1, env.action_space.shape[0]))\n",
    "    action = actor_critic.act(obs_seq,act_seq)\n",
    "    action = action.reshape((1, env.action_space.shape[0]))\n",
    "\n",
    "    new_state, reward, done, _ = env.step(action)\n",
    "    #reward += reward\n",
    "    #if j == (trial_len - 1):\n",
    "        #done = True\n",
    "        #print(reward)\n",
    "\n",
    "    #if (j % 5 == 0):\n",
    "    #    actor_critic.train()\n",
    "    #    actor_critic.update_target()   \n",
    "\n",
    "    new_state = new_state.reshape((1, env.observation_space.shape[0]))\n",
    "\n",
    "    #actor_critic.remember(cur_state, action, reward, new_state, done)\n",
    "    cur_state = new_state\n",
    "    act_list.append(action)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
